{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "      <th>housing_median_age</th>\n",
       "      <th>total_rooms</th>\n",
       "      <th>total_bedrooms</th>\n",
       "      <th>population</th>\n",
       "      <th>households</th>\n",
       "      <th>median_income</th>\n",
       "      <th>median_house_value</th>\n",
       "      <th>ocean_proximity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-122.23</td>\n",
       "      <td>37.88</td>\n",
       "      <td>41.0</td>\n",
       "      <td>880.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>322.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>8.3252</td>\n",
       "      <td>452600.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-122.22</td>\n",
       "      <td>37.86</td>\n",
       "      <td>21.0</td>\n",
       "      <td>7099.0</td>\n",
       "      <td>1106.0</td>\n",
       "      <td>2401.0</td>\n",
       "      <td>1138.0</td>\n",
       "      <td>8.3014</td>\n",
       "      <td>358500.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-122.24</td>\n",
       "      <td>37.85</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1467.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>496.0</td>\n",
       "      <td>177.0</td>\n",
       "      <td>7.2574</td>\n",
       "      <td>352100.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-122.25</td>\n",
       "      <td>37.85</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1274.0</td>\n",
       "      <td>235.0</td>\n",
       "      <td>558.0</td>\n",
       "      <td>219.0</td>\n",
       "      <td>5.6431</td>\n",
       "      <td>341300.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-122.25</td>\n",
       "      <td>37.85</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1627.0</td>\n",
       "      <td>280.0</td>\n",
       "      <td>565.0</td>\n",
       "      <td>259.0</td>\n",
       "      <td>3.8462</td>\n",
       "      <td>342200.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   longitude  latitude  housing_median_age  total_rooms  total_bedrooms  \\\n",
       "0    -122.23     37.88                41.0        880.0           129.0   \n",
       "1    -122.22     37.86                21.0       7099.0          1106.0   \n",
       "2    -122.24     37.85                52.0       1467.0           190.0   \n",
       "3    -122.25     37.85                52.0       1274.0           235.0   \n",
       "4    -122.25     37.85                52.0       1627.0           280.0   \n",
       "\n",
       "   population  households  median_income  median_house_value ocean_proximity  \n",
       "0       322.0       126.0         8.3252            452600.0        NEAR BAY  \n",
       "1      2401.0      1138.0         8.3014            358500.0        NEAR BAY  \n",
       "2       496.0       177.0         7.2574            352100.0        NEAR BAY  \n",
       "3       558.0       219.0         5.6431            341300.0        NEAR BAY  \n",
       "4       565.0       259.0         3.8462            342200.0        NEAR BAY  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "def load_housing_data(housing_path = './'):\n",
    "    csv_path = os.path.join(housing_path, 'housing.csv')\n",
    "    return pd.read_csv(csv_path)\n",
    "\n",
    "housing = load_housing_data()\n",
    "housing.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_matrix = housing.corr()\n",
    "\n",
    "Feature_in_order = corr_matrix['median_house_value'].abs().sort_values(ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.327835</td>\n",
       "      <td>1.052548</td>\n",
       "      <td>0.982143</td>\n",
       "      <td>-0.804819</td>\n",
       "      <td>-0.972476</td>\n",
       "      <td>-0.974429</td>\n",
       "      <td>-0.977033</td>\n",
       "      <td>2.344766</td>\n",
       "      <td>2.129631</td>\n",
       "      <td>-0.066145</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.322844</td>\n",
       "      <td>1.043185</td>\n",
       "      <td>-0.607019</td>\n",
       "      <td>2.045890</td>\n",
       "      <td>1.357143</td>\n",
       "      <td>0.861439</td>\n",
       "      <td>1.669961</td>\n",
       "      <td>2.332238</td>\n",
       "      <td>1.314156</td>\n",
       "      <td>-0.411672</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.332827</td>\n",
       "      <td>1.038503</td>\n",
       "      <td>1.856182</td>\n",
       "      <td>-0.535746</td>\n",
       "      <td>-0.827024</td>\n",
       "      <td>-0.820777</td>\n",
       "      <td>-0.843637</td>\n",
       "      <td>1.782699</td>\n",
       "      <td>1.258693</td>\n",
       "      <td>-0.246922</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.337818</td>\n",
       "      <td>1.038503</td>\n",
       "      <td>1.856182</td>\n",
       "      <td>-0.624215</td>\n",
       "      <td>-0.719723</td>\n",
       "      <td>-0.766028</td>\n",
       "      <td>-0.733781</td>\n",
       "      <td>0.932968</td>\n",
       "      <td>1.165100</td>\n",
       "      <td>0.122878</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.337818</td>\n",
       "      <td>1.038503</td>\n",
       "      <td>1.856182</td>\n",
       "      <td>-0.462404</td>\n",
       "      <td>-0.612423</td>\n",
       "      <td>-0.759847</td>\n",
       "      <td>-0.629157</td>\n",
       "      <td>-0.012881</td>\n",
       "      <td>1.172900</td>\n",
       "      <td>1.003418</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1         2         3         4         5         6   \\\n",
       "0 -1.327835  1.052548  0.982143 -0.804819 -0.972476 -0.974429 -0.977033   \n",
       "1 -1.322844  1.043185 -0.607019  2.045890  1.357143  0.861439  1.669961   \n",
       "2 -1.332827  1.038503  1.856182 -0.535746 -0.827024 -0.820777 -0.843637   \n",
       "3 -1.337818  1.038503  1.856182 -0.624215 -0.719723 -0.766028 -0.733781   \n",
       "4 -1.337818  1.038503  1.856182 -0.462404 -0.612423 -0.759847 -0.629157   \n",
       "\n",
       "         7         8         9    10   11   12   13   14  \n",
       "0  2.344766  2.129631 -0.066145  0.0  0.0  0.0  1.0  0.0  \n",
       "1  2.332238  1.314156 -0.411672  0.0  0.0  0.0  1.0  0.0  \n",
       "2  1.782699  1.258693 -0.246922  0.0  0.0  0.0  1.0  0.0  \n",
       "3  0.932968  1.165100  0.122878  0.0  0.0  0.0  1.0  0.0  \n",
       "4 -0.012881  1.172900  1.003418  0.0  0.0  0.0  1.0  0.0  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "No1_ix = list(housing.columns).index(Feature_in_order.index[1]) # 获取最相关的重要参数的index\n",
    "median_house_value_ix = list(housing.columns).index(\"median_house_value\")\n",
    "\n",
    "class CombinedAttributesAdder(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, add_No1Ix_per_medianHouseVlaue = True):\n",
    "        self.k = 3  # 指定选取前k个重要参数\n",
    "        self.add_No1Ix_per_medianHouseVlaue = add_No1Ix_per_medianHouseVlaue\n",
    "        self.MostImportant = [] # list用来存储前k个重要参数的label\n",
    "    def fit(self, X, y = None):\n",
    "        self.MostImportant = [Feature_in_order.index[i] for i in range(1, self.k + 1)] # 获取前k个重要参数\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y = None):\n",
    "        if self.add_No1Ix_per_medianHouseVlaue:\n",
    "            No1Ix_per_medianHouseVlaue = X[:, median_house_value_ix] / X[:, No1_ix]\n",
    "            return np.c_[X, No1Ix_per_medianHouseVlaue]\n",
    "        else:\n",
    "            return X\n",
    "        \n",
    "from sklearn.impute import SimpleImputer \n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.base import TransformerMixin\n",
    "\n",
    "class DataFrameSelector(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, attribute_names):\n",
    "        self.attribute_names = attribute_names\n",
    "    def fit(self, X, y = None):\n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[self.attribute_names].values\n",
    "    \n",
    "class MyLabelBinarizer(TransformerMixin):\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        self.encoder = LabelBinarizer(*args, **kwargs)\n",
    "    def fit(self, x, y = 0):\n",
    "        self.encoder.fit(x)\n",
    "        return self\n",
    "    def transform(self, x, y = 0):\n",
    "        return self.encoder.transform(x)\n",
    "\n",
    "housing_num = housing.drop('ocean_proximity', axis = 1)\n",
    "num_attribs = list(housing_num)\n",
    "    \n",
    "num_pipeline = Pipeline([\n",
    "    ('selector', DataFrameSelector(num_attribs)),\n",
    "    ('imputer', SimpleImputer(strategy = \"median\")), # 数据清洗中位数替代\n",
    "    ('attribs_adder', CombinedAttributesAdder()), # 整合新的数据类型\n",
    "    ('std_scaler', StandardScaler())\n",
    "])\n",
    "\n",
    "cat_attribs = ['ocean_proximity']\n",
    "\n",
    "cat_pipeline = Pipeline([\n",
    "    ('selector', DataFrameSelector(cat_attribs)),\n",
    "    ('LabelBinarizer', MyLabelBinarizer()),\n",
    "])\n",
    "\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "\n",
    "full_pipeline = FeatureUnion(transformer_list = [\n",
    "    (\"num_pipline\", num_pipeline),\n",
    "    (\"cat_pipline\", cat_pipeline),\n",
    "])\n",
    "\n",
    "housing_finished = full_pipeline.fit_transform(housing)\n",
    "housing_prepared = pd.DataFrame(housing_finished)\n",
    "housing_prepared.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
